package top.igotcha.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.jupiter.api.Test;

import java.net.URI;

/**
 * @time 2021/1/10
 * @auth Gotcha
 * @describe
 */
public class HdfsListFile {
    //递归调用
    @Test
    public void listFile() throws Exception{
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.127.110:8020"), new Configuration());
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
        for (FileStatus fileStatus : fileStatuses) {
            if(fileStatus.isDirectory()){
                Path path = fileStatus.getPath();
                listAllFiles(fileSystem,path);
            }else{
                System.out.println("文件路径为"+fileStatus.getPath().toString());
            }
        }
    }
    public void listAllFiles(FileSystem fileSystem,Path path) throws  Exception{
        FileStatus[] fileStatuses = fileSystem.listStatus(path);
        for (FileStatus fileStatus : fileStatuses) {
            if(fileStatus.isDirectory()){
                listAllFiles(fileSystem,fileStatus.getPath());
            }else{
                Path path1 = fileStatus.getPath();
                System.out.println("文件路径为"+path1);
            }
        }
    }

    @Test
    public void listMyFiles()throws Exception{
        //获取fileSystem类
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.127.110:8020"), new Configuration());
        //获取RemoteIterator 得到所有的文件或者文件夹，第一个参数指定遍历的路径，第二个参数表示是否要递归遍历
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fileSystem.listFiles(new Path("/"), true);
        while (locatedFileStatusRemoteIterator.hasNext()){
            //获取每个文件详细信息
            LocatedFileStatus fileStatus = locatedFileStatusRemoteIterator.next();
            //打印文件路径
            System.out.println(fileStatus.getPath().toString());
            //获取文件Block信息
            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            //打印每个文件副本存储位置
            for (BlockLocation blockLocation : blockLocations) {
                String[] hosts = blockLocation.getHosts();
                for (String host : hosts) {
                    System.out.println(host);
                }
            }
        }
        fileSystem.close();
    }
}
